package core.proto.search;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Iterator;

import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Source;
import net.htmlparser.jericho.StartTag;
import net.htmlparser.jericho.TextExtractor;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;

import core.proto.Util;

/**
 * Utility class to create index file for Lucene. Invokable on the command line.  
 */
public class LuceneIndexCreationUtil {

	public static final String FILENAME_FIELD = "filename";
	public static final String CONTENT_FIELD = "content";

	public static void main(String[] args){
		if (args.length != 2){
			System.err.println("Usage: java " + LuceneIndexCreationUtil.class.toString() + " <indexDir> <htmlFolder>");
			System.exit(-1);
		}

		LuceneIndexCreationUtil indexer = new LuceneIndexCreationUtil();

		// Remove any existing index. 
		File indexDir = new File(args[0]);
		if (indexDir.exists()){
			try {
				FileUtils.deleteDirectory(indexDir);
			} catch (IOException e) {
			}
		}
		
		try{
			indexer.index(args[0], args[1]);
		}catch(Exception e){
			throw new Error(e);
		}
	}
	/**
	 * Flattens given html file to text for processing by Lucene. 
	 * Uses the TextExtractor class of the Jericho html parser library to extract the 
	 * textual content of the html files. Then add it to the Lucene Document object 
	 * as a field. 
	 * @param htmlFile
	 * @return
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	Document htmlToDoc(File htmlFile) throws MalformedURLException, IOException{
		Document doc = new Document();
		// name, value, store?, index?
		doc.add(new Field(FILENAME_FIELD, FilenameUtils.getBaseName(htmlFile.getAbsolutePath()), true, false, false));
		doc.add(new Field(CONTENT_FIELD, HtmlTextExtractor.extract(htmlFile), false, true, true));
		return doc;
	}

	static Analyzer createAnalyzer(){
		return new StandardAnalyzer();
	}
	
	/**
	 * Indexes all the html files in the given folder and outputs to the given index. 
	 * @param indexDirPath Where to write index. 
	 * @param htmlFolderPath Where to get html files. 
	 * @throws CorruptIndexException
	 * @throws LockObtainFailedException
	 * @throws IOException
	 */
	void index(String indexDirPath, String htmlFolderPath) throws IOException {
		Directory indexDir = FSDirectory.getDirectory(new File(indexDirPath), true);
		IndexWriter iWriter = new IndexWriter(indexDir, createAnalyzer(), true);
		try{
			String[] htmlExtension = {"html", "htm"};
			Iterator<File> fileIter = FileUtils.iterateFiles(new File(htmlFolderPath), htmlExtension, true);
			while(fileIter.hasNext()){
				File htmlFile = fileIter.next();
				iWriter.addDocument(htmlToDoc(htmlFile));
			}
			iWriter.optimize();
		}finally{
			iWriter.close();
		}
	}	
}
